#!/usr/bin/python
#coding: utf-8

import sys
sys.path.append("..")

import time
import random
import lxml.html
from Config.Config import ProxyUrl, TimeSleep, USER_AGENTS
from Downloader.ProxyDownloader import DownloaderPageHtml
from MySql_InterFace.ProxyMySql import ProxyObject
from Logging.ProxyLogging import log_Main

class ProxyAnalysis(object):
    u"代理IP的网页解析"
    def __init__(self):
        self.results = set()
        self.po = ProxyObject()
        self.create = self.po.Create()

    def ProxyMain(self, ):
        try:
            html = DownloaderPageHtml(ProxyUrl, USER_AGENTS[random.randint(0, len(USER_AGENTS) - 1)]).Downloader(3)
            tree = lxml.html.fromstring(html)
            self.FindIP(tree)
            tree = tree.cssselect("div#listnav > ul > li > a")
            # 获取代理ip页面的数量
            page = tree[-1].text_content()
            # print page
            # print ProxyUrl
            # 从2开始，链接中不适用页码的时候默认是爬取第一页
            for i in range(2, (int)(page) + 1):
                # 休眠1s
                time.sleep(TimeSleep)
                html = DownloaderPageHtml(ProxyUrl, USER_AGENTS[random.randint(0, len(USER_AGENTS) - 1)]).Downloader(3)
                tree = lxml.html.fromstring(html)
                self.FindIP(tree)
            # 把代理IP写入数据库
            print u"把代理ip写入数据库"
            self.po.Insert(self.results)
        except Exception as e:
            print e

    def FindIP(self, tree):
        u"获取代理ip"
        tree = tree.cssselect("table > tbody > tr")
        for t in tree:
            ip = t[0].text_content()
            port = t[1].text_content()
            type = t[3].text_content()
            res = (ip, port, type)
            self.results.add(res)

    def Close(self):
        # self.po.Close()
        pass

def ProxyManagerMain():
    p = ProxyAnalysis()
    p.ProxyMain()
    p.Close()
    # log_Main(u"代理ip爬取结束！！！", "info")

    CONTENT = """
        代理ip爬取完成
        """
    from SendEmail.SendEmail import SendMain
    # SendMain(CONTENT)

if __name__ == "__main__":
    ProxyManagerMain()